# -*- coding: utf-8 -*-

import logging
import os
import time
import datetime

from scrapy.crawler import CrawlerProcess
from django.core.management.base import BaseCommand, CommandError
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from socar import settings
from selenium import webdriver

city_list = [
    'https://www.autohome.com.cn/beijing/cheshi/',
    'https://www.autohome.com.cn/tianjin/cheshi/',
    'https://www.autohome.com.cn/shijiazhuang/cheshi/',
    'https://www.autohome.com.cn/tangshan/cheshi/',
    'https://www.autohome.com.cn/handan/cheshi/',
    'https://www.autohome.com.cn/baoding/cheshi/',
    'https://www.autohome.com.cn/xingtai/cheshi/',
    'https://www.autohome.com.cn/cangzhou/cheshi/',
    'https://www.autohome.com.cn/qinhuangdao/cheshi/',
    'https://www.autohome.com.cn/zhangjiakou/cheshi/',
    'https://www.autohome.com.cn/chengde/cheshi/',
    'https://www.autohome.com.cn/langfang/cheshi/',
    'https://www.autohome.com.cn/hengshui/cheshi/',
    'https://www.autohome.com.cn/huhehaote/cheshi/',
    'https://www.autohome.com.cn/eerduosi/cheshi/',
    'https://www.autohome.com.cn/baotou/cheshi/',
    'https://www.autohome.com.cn/chifeng/cheshi/',
    'https://www.autohome.com.cn/hulunbeier/cheshi/',
    'https://www.autohome.com.cn/tongliao/cheshi/',
    'https://www.autohome.com.cn/bayannaoer/cheshi/',
    'https://www.autohome.com.cn/xilinguolemeng/cheshi/',
    'https://www.autohome.com.cn/wuhai/cheshi/',
    'https://www.autohome.com.cn/wulanchabu/cheshi/',
    'https://www.autohome.com.cn/xinganmeng/cheshi/',
    'https://www.autohome.com.cn/jinan/cheshi/',
    'https://www.autohome.com.cn/qingdao/cheshi/',
    'https://www.autohome.com.cn/weifang/cheshi/',
    'https://www.autohome.com.cn/zibo/cheshi/',
    'https://www.autohome.com.cn/yantai/cheshi/',
    'https://www.autohome.com.cn/linyi/cheshi/',
    'https://www.autohome.com.cn/jining/cheshi/',
    'https://www.autohome.com.cn/taian/cheshi/',
    'https://www.autohome.com.cn/binzhou/cheshi/',
    'https://www.autohome.com.cn/liaocheng/cheshi/',
    'https://www.autohome.com.cn/dezhou/cheshi/',
    'https://www.autohome.com.cn/dongying/cheshi/',
    'https://www.autohome.com.cn/weihai/cheshi/',
    'https://www.autohome.com.cn/rizhao/cheshi/',
    'https://www.autohome.com.cn/zaozhuang/cheshi/',
    'https://www.autohome.com.cn/laiwu/cheshi/',
    'https://www.autohome.com.cn/heze/cheshi/',
    'https://www.autohome.com.cn/taiyuan/cheshi/',
    'https://www.autohome.com.cn/datong/cheshi/',
    'https://www.autohome.com.cn/changzhi/cheshi/',
    'https://www.autohome.com.cn/jincheng/cheshi/',
    'https://www.autohome.com.cn/lvliang/cheshi/',
    'https://www.autohome.com.cn/jinzhong/cheshi/',
    'https://www.autohome.com.cn/linfen/cheshi/',
    'https://www.autohome.com.cn/yuncheng/cheshi/',
    'https://www.autohome.com.cn/shuozhou/cheshi/',
    'https://www.autohome.com.cn/yangquan/cheshi/',
    'https://www.autohome.com.cn/xinzhou/cheshi/',
    'https://www.autohome.com.cn/shanghai/cheshi/',
    'https://www.autohome.com.cn/hefei/cheshi/',
    'https://www.autohome.com.cn/su_zhou/cheshi/',
    'https://www.autohome.com.cn/huaibei/cheshi/',
    'https://www.autohome.com.cn/fu_yang/cheshi/',
    'https://www.autohome.com.cn/bangbu/cheshi/',
    'https://www.autohome.com.cn/huainan/cheshi/',
    'https://www.autohome.com.cn/chuzhou/cheshi/',
    'https://www.autohome.com.cn/maanshan/cheshi/',
    'https://www.autohome.com.cn/wuhu/cheshi/',
    'https://www.autohome.com.cn/tongling/cheshi/',
    'https://www.autohome.com.cn/anqing/cheshi/',
    'https://www.autohome.com.cn/huangshan/cheshi/',
    'https://www.autohome.com.cn/liuan/cheshi/',
    'https://www.autohome.com.cn/chizhou/cheshi/',
    'https://www.autohome.com.cn/xuancheng/cheshi/',
    'https://www.autohome.com.cn/bozhou/cheshi/',
    'https://www.autohome.com.cn/nanjing/cheshi/',
    'https://www.autohome.com.cn/suzhou/cheshi/',
    'https://www.autohome.com.cn/xuzhou/cheshi/',
    'https://www.autohome.com.cn/yancheng/cheshi/',
    'https://www.autohome.com.cn/yangzhou/cheshi/',
    'https://www.autohome.com.cn/tai_zhou/cheshi/',
    'https://www.autohome.com.cn/nantong/cheshi/',
    'https://www.autohome.com.cn/changzhou/cheshi/',
    'https://www.autohome.com.cn/wuxi/cheshi/',
    'https://www.autohome.com.cn/lianyungang/cheshi/',
    'https://www.autohome.com.cn/suqian/cheshi/',
    'https://www.autohome.com.cn/huaian/cheshi/',
    'https://www.autohome.com.cn/zhenjiang/cheshi/',
    'https://www.autohome.com.cn/hangzhou/cheshi/',
    'https://www.autohome.com.cn/ningbo/cheshi/',
    'https://www.autohome.com.cn/jiaxing/cheshi/',
    'https://www.autohome.com.cn/shaoxing/cheshi/',
    'https://www.autohome.com.cn/jinhua/cheshi/',
    'https://www.autohome.com.cn/taizhou/cheshi/',
    'https://www.autohome.com.cn/wenzhou/cheshi/',
    'https://www.autohome.com.cn/huzhou/cheshi/',
    'https://www.autohome.com.cn/quzhou/cheshi/',
    'https://www.autohome.com.cn/lishui/cheshi/',
    'https://www.autohome.com.cn/zhoushan/cheshi/',
    'https://www.autohome.com.cn/fuzhou/cheshi/',
    'https://www.autohome.com.cn/xiamen/cheshi/',
    'https://www.autohome.com.cn/quanzhou/cheshi/',
    'https://www.autohome.com.cn/nanping/cheshi/',
    'https://www.autohome.com.cn/sanming/cheshi/',
    'https://www.autohome.com.cn/putian/cheshi/',
    'https://www.autohome.com.cn/zhangzhou/cheshi/',
    'https://www.autohome.com.cn/longyan/cheshi/',
    'https://www.autohome.com.cn/ningde/cheshi/',
    'https://www.autohome.com.cn/guangzhou/cheshi/',
    'https://www.autohome.com.cn/shenzhen/cheshi/',
    'https://www.autohome.com.cn/foshan/cheshi/',
    'https://www.autohome.com.cn/huizhou/cheshi/',
    'https://www.autohome.com.cn/dongguan/cheshi/',
    'https://www.autohome.com.cn/zhongshan/cheshi/',
    'https://www.autohome.com.cn/qingyuan/cheshi/',
    'https://www.autohome.com.cn/shaoguan/cheshi/',
    'https://www.autohome.com.cn/heyuan/cheshi/',
    'https://www.autohome.com.cn/meizhou/cheshi/',
    'https://www.autohome.com.cn/chaozhou/cheshi/',
    'https://www.autohome.com.cn/shantou/cheshi/',
    'https://www.autohome.com.cn/jieyang/cheshi/',
    'https://www.autohome.com.cn/zhuhai/cheshi/',
    'https://www.autohome.com.cn/jiangmen/cheshi/',
    'https://www.autohome.com.cn/zhaoqing/cheshi/',
    'https://www.autohome.com.cn/maoming/cheshi/',
    'https://www.autohome.com.cn/zhanjiang/cheshi/',
    'https://www.autohome.com.cn/shanwei/cheshi/',
    'https://www.autohome.com.cn/yunfu/cheshi/',
    'https://www.autohome.com.cn/yangjiang/cheshi/',
    'https://www.autohome.com.cn/nanning/cheshi/',
    'https://www.autohome.com.cn/guilin/cheshi/',
    'https://www.autohome.com.cn/liuzhou/cheshi/',
    'https://www.autohome.com.cn/guigang/cheshi/',
    'https://www.autohome.com.cn/yu_lin/cheshi/',
    'https://www.autohome.com.cn/baise/cheshi/',
    'https://www.autohome.com.cn/wuzhou/cheshi/',
    'https://www.autohome.com.cn/qinzhou/cheshi/',
    'https://www.autohome.com.cn/beihai/cheshi/',
    'https://www.autohome.com.cn/fangchenggang/cheshi/',
    'https://www.autohome.com.cn/chongzuo/cheshi/',
    'https://www.autohome.com.cn/hechi/cheshi/',
    'https://www.autohome.com.cn/laibin/cheshi/',
    'https://www.autohome.com.cn/hezhou/cheshi/',
    'https://www.autohome.com.cn/haikou/cheshi/',
    'https://www.autohome.com.cn/sanya/cheshi/',
    'https://www.autohome.com.cn/qionghai/cheshi/',
    'https://www.autohome.com.cn/danzhou/cheshi/',
    'https://www.autohome.com.cn/nanchang/cheshi/',
    'https://www.autohome.com.cn/ganzhou/cheshi/',
    'https://www.autohome.com.cn/jiujiang/cheshi/',
    'https://www.autohome.com.cn/shangrao/cheshi/',
    'https://www.autohome.com.cn/yi_chun/cheshi/',
    'https://www.autohome.com.cn/jingdezhen/cheshi/',
    'https://www.autohome.com.cn/yingtan/cheshi/',
    'https://www.autohome.com.cn/xinyu/cheshi/',
    'https://www.autohome.com.cn/ping_xiang/cheshi/',
    'https://www.autohome.com.cn/fu_zhou/cheshi/',
    'https://www.autohome.com.cn/jian/cheshi/',
    'https://www.autohome.com.cn/zhengzhou/cheshi/',
    'https://www.autohome.com.cn/luoyang/cheshi/',
    'https://www.autohome.com.cn/xinxiang/cheshi/',
    'https://www.autohome.com.cn/kaifeng/cheshi/',
    'https://www.autohome.com.cn/pingdingshan/cheshi/',
    'https://www.autohome.com.cn/anyang/cheshi/',
    'https://www.autohome.com.cn/jiaozuo/cheshi/',
    'https://www.autohome.com.cn/puyang/cheshi/',
    'https://www.autohome.com.cn/xuchang/cheshi/',
    'https://www.autohome.com.cn/luohe/cheshi/',
    'https://www.autohome.com.cn/sanmenxia/cheshi/',
    'https://www.autohome.com.cn/nanyang/cheshi/',
    'https://www.autohome.com.cn/shangqiu/cheshi/',
    'https://www.autohome.com.cn/zhoukou/cheshi/',
    'https://www.autohome.com.cn/zhumadian/cheshi/',
    'https://www.autohome.com.cn/xinyang/cheshi/',
    'https://www.autohome.com.cn/hebi/cheshi/',
    'https://www.autohome.com.cn/jiyuanshi/cheshi/',
    'https://www.autohome.com.cn/wuhan/cheshi/',
    'https://www.autohome.com.cn/shiyan/cheshi/',
    'https://www.autohome.com.cn/xiangyang/cheshi/',
    'https://www.autohome.com.cn/xiaogan/cheshi/',
    'https://www.autohome.com.cn/huangshi/cheshi/',
    'https://www.autohome.com.cn/jingzhou/cheshi/',
    'https://www.autohome.com.cn/yichang/cheshi/',
    'https://www.autohome.com.cn/enshi/cheshi/',
    'https://www.autohome.com.cn/jingmen/cheshi/',
    'https://www.autohome.com.cn/huanggang/cheshi/',
    'https://www.autohome.com.cn/ezhou/cheshi/',
    'https://www.autohome.com.cn/xianning/cheshi/',
    'https://www.autohome.com.cn/suizhou/cheshi/',
    'https://www.autohome.com.cn/xiantao/cheshi/',
    'https://www.autohome.com.cn/tianmen/cheshi/',
    'https://www.autohome.com.cn/qianjiang/cheshi/',
    'https://www.autohome.com.cn/changsha/cheshi/',
    'https://www.autohome.com.cn/changde/cheshi/',
    'https://www.autohome.com.cn/yiyang/cheshi/',
    'https://www.autohome.com.cn/yueyang/cheshi/',
    'https://www.autohome.com.cn/zhuzhou/cheshi/',
    'https://www.autohome.com.cn/xiangtan/cheshi/',
    'https://www.autohome.com.cn/hengyang/cheshi/',
    'https://www.autohome.com.cn/chenzhou/cheshi/',
    'https://www.autohome.com.cn/shaoyang/cheshi/',
    'https://www.autohome.com.cn/huaihua/cheshi/',
    'https://www.autohome.com.cn/loudi/cheshi/',
    'https://www.autohome.com.cn/zhangjiajie/cheshi/',
    'https://www.autohome.com.cn/yongzhou/cheshi/',
    'https://www.autohome.com.cn/xiangxi/cheshi/',
    'https://www.autohome.com.cn/haerbin/cheshi/',
    'https://www.autohome.com.cn/qiqihaer/cheshi/',
    'https://www.autohome.com.cn/daqing/cheshi/',
    'https://www.autohome.com.cn/yichun/cheshi/',
    'https://www.autohome.com.cn/jiamusi/cheshi/',
    'https://www.autohome.com.cn/heihe/cheshi/',
    'https://www.autohome.com.cn/hegang/cheshi/',
    'https://www.autohome.com.cn/shuangyashan/cheshi/',
    'https://www.autohome.com.cn/qitaihe/cheshi/',
    'https://www.autohome.com.cn/jixi/cheshi/',
    'https://www.autohome.com.cn/mudanjiang/cheshi/',
    'https://www.autohome.com.cn/suihua/cheshi/',
    'https://www.autohome.com.cn/daxinganling/cheshi/',
    'https://www.autohome.com.cn/changchun/cheshi/',
    'https://www.autohome.com.cn/jilinshi/cheshi/',
    'https://www.autohome.com.cn/songyuan/cheshi/',
    'https://www.autohome.com.cn/baicheng/cheshi/',
    'https://www.autohome.com.cn/siping/cheshi/',
    'https://www.autohome.com.cn/liaoyuan/cheshi/',
    'https://www.autohome.com.cn/tonghua/cheshi/',
    'https://www.autohome.com.cn/baishan/cheshi/',
    'https://www.autohome.com.cn/yanbian/cheshi/',
    'https://www.autohome.com.cn/shenyang/cheshi/',
    'https://www.autohome.com.cn/dalian/cheshi/',
    'https://www.autohome.com.cn/chaoyang/cheshi/',
    'https://www.autohome.com.cn/fuxin/cheshi/',
    'https://www.autohome.com.cn/anshan/cheshi/',
    'https://www.autohome.com.cn/yingkou/cheshi/',
    'https://www.autohome.com.cn/panjin/cheshi/',
    'https://www.autohome.com.cn/jinzhou/cheshi/',
    'https://www.autohome.com.cn/huludao/cheshi/',
    'https://www.autohome.com.cn/tieling/cheshi/',
    'https://www.autohome.com.cn/fushun/cheshi/',
    'https://www.autohome.com.cn/benxi/cheshi/',
    'https://www.autohome.com.cn/liaoyang/cheshi/',
    'https://www.autohome.com.cn/dandong/cheshi/',
    'https://www.autohome.com.cn/chongqing/cheshi/',
    'https://www.autohome.com.cn/guiyang/cheshi/',
    'https://www.autohome.com.cn/liupanshui/cheshi/',
    'https://www.autohome.com.cn/zunyi/cheshi/',
    'https://www.autohome.com.cn/anshun/cheshi/',
    'https://www.autohome.com.cn/bijie/cheshi/',
    'https://www.autohome.com.cn/tongren/cheshi/',
    'https://www.autohome.com.cn/kaili/cheshi/',
    'https://www.autohome.com.cn/duyunshi/cheshi/',
    'https://www.autohome.com.cn/xingyishi/cheshi/',
    'https://www.autohome.com.cn/chengdu/cheshi/',
    'https://www.autohome.com.cn/guangyuan/cheshi/',
    'https://www.autohome.com.cn/mianyang/cheshi/',
    'https://www.autohome.com.cn/deyang/cheshi/',
    'https://www.autohome.com.cn/nanchong/cheshi/',
    'https://www.autohome.com.cn/guangan/cheshi/',
    'https://www.autohome.com.cn/suining/cheshi/',
    'https://www.autohome.com.cn/leshan/cheshi/',
    'https://www.autohome.com.cn/zigong/cheshi/',
    'https://www.autohome.com.cn/luzhou/cheshi/',
    'https://www.autohome.com.cn/yibin/cheshi/',
    'https://www.autohome.com.cn/bazhong/cheshi/',
    'https://www.autohome.com.cn/dazhou/cheshi/',
    'https://www.autohome.com.cn/ziyang/cheshi/',
    'https://www.autohome.com.cn/meishan/cheshi/',
    'https://www.autohome.com.cn/yaan/cheshi/',
    'https://www.autohome.com.cn/liangshan/cheshi/',
    'https://www.autohome.com.cn/neijiang/cheshi/',
    'https://www.autohome.com.cn/panzhihua/cheshi/',
    'https://www.autohome.com.cn/lasa/cheshi/',
    'https://www.autohome.com.cn/rikaze/cheshi/',
    'https://www.autohome.com.cn/kunming/cheshi/',
    'https://www.autohome.com.cn/dali/cheshi/',
    'https://www.autohome.com.cn/qujing/cheshi/',
    'https://www.autohome.com.cn/yuxi/cheshi/',
    'https://www.autohome.com.cn/lijiang/cheshi/',
    'https://www.autohome.com.cn/chuxiong/cheshi/',
    'https://www.autohome.com.cn/honghe/cheshi/',
    'https://www.autohome.com.cn/zhaotong/cheshi/',
    'https://www.autohome.com.cn/lincang/cheshi/',
    'https://www.autohome.com.cn/baoshan/cheshi/',
    'https://www.autohome.com.cn/wenshan/cheshi/',
    'https://www.autohome.com.cn/puer/cheshi/',
    'https://www.autohome.com.cn/diqing/cheshi/',
    'https://www.autohome.com.cn/xishuangbanna/cheshi/',
    'https://www.autohome.com.cn/dehong/cheshi/',
    'https://www.autohome.com.cn/lanzhou/cheshi/',
    'https://www.autohome.com.cn/tianshui/cheshi/',
    'https://www.autohome.com.cn/jiuquan/cheshi/',
    'https://www.autohome.com.cn/zhangye/cheshi/',
    'https://www.autohome.com.cn/jiayuguan/cheshi/',
    'https://www.autohome.com.cn/jinchang/cheshi/',
    'https://www.autohome.com.cn/baiyin/cheshi/',
    'https://www.autohome.com.cn/wuwei/cheshi/',
    'https://www.autohome.com.cn/qingyang/cheshi/',
    'https://www.autohome.com.cn/pingliang/cheshi/',
    'https://www.autohome.com.cn/dingxi/cheshi/',
    'https://www.autohome.com.cn/longnan/cheshi/',
    'https://www.autohome.com.cn/linxia/cheshi/',
    'https://www.autohome.com.cn/yinchuan/cheshi/',
    'https://www.autohome.com.cn/shizuishan/cheshi/',
    'https://www.autohome.com.cn/wuzhong/cheshi/',
    'https://www.autohome.com.cn/zhongwei/cheshi/',
    'https://www.autohome.com.cn/guyuan/cheshi/',
    'https://www.autohome.com.cn/xining/cheshi/',
    'https://www.autohome.com.cn/haidong/cheshi/',
    'https://www.autohome.com.cn/haixi/cheshi/',
    'https://www.autohome.com.cn/xian/cheshi/',
    'https://www.autohome.com.cn/yanan/cheshi/',
    'https://www.autohome.com.cn/weinan/cheshi/',
    'https://www.autohome.com.cn/xianyang/cheshi/',
    'https://www.autohome.com.cn/baoji/cheshi/',
    'https://www.autohome.com.cn/hanzhong/cheshi/',
    'https://www.autohome.com.cn/yulin/cheshi/',
    'https://www.autohome.com.cn/tongchuan/cheshi/',
    'https://www.autohome.com.cn/shangluo/cheshi/',
    'https://www.autohome.com.cn/ankang/cheshi/',
    'https://www.autohome.com.cn/wulumuqi/cheshi/',
    'https://www.autohome.com.cn/akesu/cheshi/',
    'https://www.autohome.com.cn/changji/cheshi/',
    'https://www.autohome.com.cn/yili/cheshi/',
    'https://www.autohome.com.cn/bayinguoleng/cheshi/',
    'https://www.autohome.com.cn/kelamayi/cheshi/',
    'https://www.autohome.com.cn/shihezi/cheshi/',
    'https://www.autohome.com.cn/alaer/cheshi/',
    'https://www.autohome.com.cn/tumushuke/cheshi/',
    'https://www.autohome.com.cn/wujiaqu/cheshi/',
    'https://www.autohome.com.cn/kashen/cheshi/',
    'https://www.autohome.com.cn/hetian/cheshi/',
    'https://www.autohome.com.cn/tulufan/cheshi/',
    'https://www.autohome.com.cn/hami/cheshi/',
    'https://www.autohome.com.cn/tacheng/cheshi/',
    'https://www.autohome.com.cn/aletai/cheshi/',
]


# python manage.py run_bbs_spider
class Command(BaseCommand):
    help = 'run kb_mobile spider'

    def add_arguments(self, parser):
        parser.add_argument('-a', action = 'append', dest = 'spargs', default = [],
                            help = 'set spider argument (may be repeated)')

    # 必须实现的方法
    def handle(self, *args, **options):
        os.chdir(settings.BASE_DIR)

        spargs = arglist_to_dict(options['spargs'])

        # configure_logging(install_root_handler = False)
        # logging.basicConfig(
        #     filename = 'log/kb_mobile_%s.log' % (time.strftime('%Y-%m-%d', time.localtime())),
        #     format = '%(levelname)s %(asctime)s: %(message)s',
        #     level = logging.ERROR
        # )

        for city in city_list:
            browser = webdriver.PhantomJS()
            browser.set_window_size(5120, 2880)
            browser.get(city)

            d = webdriver.PhantomJS()
            d.set_page_load_timeout(10)
            d.set_script_timeout(10)

            try:
                d.get(city)
            except:
                d.execute_script('window.stop()')

            print('time:%s spargs:%s' % (str(datetime.datetime.now()), spargs))

            runspider(spargs = spargs, browser = browser)


def runspider(spargs, browser):
    name = spargs.get('name', 'subsidy')
    spargs['browser'] = browser

    logging.debug('get_project_settings().attributes: %s' % get_project_settings().attributes['SPIDER_MODULES'])
    process = CrawlerProcess(get_project_settings())
    start_time = time.time()
    try:
        logging.info('进入爬虫:%s time:%s' % (name, str(datetime.datetime.now())))
        process.crawl(name, **spargs)
        process.start()
    except Exception as e:
        process.stop()
        logging.error('ERROR runspider exception msg:%s spargs:%s' % (e, spargs))
    finally:
        logging.debug('FINISH spider spargs:%s crawl_time:%s\n\n\n\n' % (spargs, str(time.time() - start_time)))

    process.stop()





def arglist_to_dict(arglist):
    """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
    dict
    """
    return dict(x.split('=', 1) for x in arglist)

